# -*-coding:utf8-*-
__author__ = "no name"

import requests
import json
import random
import pymysql
import sys
import datetime
import time
import re
from bs4 import BeautifulSoup
#from imp import reload
from multiprocessing.dummy import Pool as ThreadPool


def datetime_to_timestamp_in_milliseconds(d):
    def current_milli_time(): return int(round(time.time() * 1000))

    return current_milli_time()


#reload(sys)


def LoadUserAgents(uafile):
    uas = []
    with open(uafile, 'rb') as uaf:
        for ua in uaf.readlines():
            if ua:
                uas.append(ua.strip()[:-1])
    random.shuffle(uas)
    return uas


uas = LoadUserAgents("user_agents.txt")
head = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3753.4 Safari/537.36',
}

# Please replace your own proxies.
proxies = {
    'http': 'http://27.191.234.69:9999',
    #'http': 'http://101.89.132.131:80',
}
time1 = time.time()

urls = ["https://price.pcauto.com.cn/cars/6/2018/1/",
        "https://price.pcauto.com.cn/cars/6/2018/2/",
        "https://price.pcauto.com.cn/cars/6/2018/3/",
        "https://price.pcauto.com.cn/cars/6/2018/4/",
        "https://price.pcauto.com.cn/cars/6/2018/5/",
        "https://price.pcauto.com.cn/cars/6/2018/6/",
        "https://price.pcauto.com.cn/cars/6/2018/7/",
        "https://price.pcauto.com.cn/cars/6/2018/8/",
        "https://price.pcauto.com.cn/cars/6/2018/9/",
        "https://price.pcauto.com.cn/cars/6/2018/10/",
        "https://price.pcauto.com.cn/cars/6/2018/11/",
        "https://price.pcauto.com.cn/cars/6/2018/12/",
        "https://price.pcauto.com.cn/cars/6/2019/1/",
        "https://price.pcauto.com.cn/cars/6/2019/2/",
        "https://price.pcauto.com.cn/cars/6/2019/3/",
        "https://price.pcauto.com.cn/cars/6/2019/4/"]

# Please change the range data by yourself.
def loadHtml(url):
    response = requests.get(url,headers=head)
    soup = BeautifulSoup(response.text, "html.parser")
    car_ul = soup.find(attrs={"class":"picList clearfix"})
    car_p = car_ul.findAll('p',attrs={"class": "tit"})
    car_price = car_ul.findAll('p',attrs={"class": "price"})
    car_market_date = car_ul.findAll('p',attrs={"class": "link"})

    car_name = []
    for item in  car_p :
        car_name.append(item.findAll('a')[0])

    name_list = []
    id_list = []
    for item in  car_name :
        name_list.append(item.get("title"))
        id_list.append(item.get("href").replace("/",""))
    price_list = []
    for item in  car_price :
        price_list.append(item.get_text().replace("\r\n",""))
    market_date_list = []
    for item in  car_market_date :
        market_date_list.append(item.get_text().replace("\r\n",""))
    
    sqlField = "INSERT INTO pcauto_car (name, price, market_date, href_id) VALUES "
    valueArr = []
    index = 0
    for item in name_list :
        flag = ","
        if index==len(name_list)-1 : flag = ""
        tempStr = '\"'+ name_list[index] + '\",'+ '\"'+ price_list[index] + '\",'+ '\"'+ market_date_list[index] + '\",'+ '\"'+ id_list[index] + '\"'
        valueArr.append("("+tempStr+")")
        index = index+1
    sqlStr = sqlField + ",".join(valueArr)
    #print(sqlStr)
    try:
        # Please write your MySQL's information.
        conn = pymysql.connect(host='localhost', user='root', passwd='admin9527', db='python', charset='utf8')
        cur = conn.cursor()
        cur.execute(sqlStr)
        conn.commit()
    except Exception as e:
        print("error" + e)



if __name__ == "__main__" :
    pool = ThreadPool(1)
    try:
        results = pool.map(loadHtml, urls)
    except Exception as e:
        print(e)
 
    pool.close()
    pool.join()


# 0_100_speed_up, measured_0_100_speed_up, measured_100_0_braking, measured_fuel_consumption, offical_fuel_consumption, offical_pure_electric_range, quality_assurance, body_image_url, body_type, body_length, body_width, body_height, body_wheelbase, body_front_wheel_pitch, body_rear_wheel_pitch, body_min_ground_clearance, body_weight, body_door_num, body_sit_num, body_tank_volume, body_reserve_compartment_volume, body_reserve_compartment_max_volume, body_reserve_compartment_size, engine_image_url, engine_model, engine_displacement, engine_intake_form, engine_max_ps, engine_max_power, engine_max_power_speed, engine_max_torque, engine_max_torque_speed, engine_cylinder_arrangement_form, engine_cylinder_num, engine_cylinder_door_num, engine_compression_ratio, engine_valve_train, engine_cylinder_diameter, engine_Itinerary, engine_specific_technology, engine_fuel_form, engine_fuel_grade, engine_fuel_supply_mode, engine_cylinder_head_material, engine_cylinder_block_material, engine_emission_standard, gearbox_name, gearbox_gear_num, gearbox_type, turn_image_url, turn_driving_mode, turn_four_wheel_drive, turn_central_differential_structure, turn_front_suspension_type, turn_back_suspension_type, turn_steering_boost_type, turn_car_body_structure, wheelbrake_image_url, wheelbrake_front_brake_type, wheelbrake_back_brake_type, wheelbrake_parking_brake_type, wheelbrake_front_tire_specification, wheelbrake_back_tire_specification, wheelbrake_spare_tire_specification, wheelbrake_spare_tire_size, offroad_approach_angle, offroad_departure_angle, offroad_longitudinal_angle_passage, offroad_climbing_angle, offroad_min_ground_clearance, offroad_min_turning_radius, offroad_max_wading_depth